// Copyright 2006 PR1ME. All Rights Reserved.
import java.io.DataInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Date;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;

/**
 * Represents an entire corpus of files.
 */
public class Corpus {

  public static Corpus createCorpus(DataInputStream input, long indexFileSize)
      throws IOException {
    // 1: Date of time of construction
    final long indexCreationDate = input.readLong();

    // 2: Variable break characters
    String varBreakChars = "";
    for (byte b = input.readByte(); b != 0; b = input.readByte()) {
      varBreakChars += (char) (b & 0xFF);
    }

    // 3: stop word file
    String stopFileName = input.readUTF();

    // 4: the construction directory
    String indexDir = input.readUTF();

    // 5: corpus stats
    long corpusChars = input.readLong();
    long corpusWords = input.readLong();
    int corpusFiles = input.readInt();

    // 6: each file path
    String[] filePaths = new String[corpusFiles];
    for (int i = 0; i < corpusFiles; ++i) {
      filePaths[i] = input.readUTF();
    }

    // 7: the corpus word list, preceded by its size
    int wordListCount = input.readInt();
    String[] wordList = new String[wordListCount];
    for (int i = 0; i < wordListCount; ++i) {
      wordList[i] = input.readUTF();
    }
    int[] wordFrequency = new int[wordListCount];

    // 8: each file's bit set
    CorpusFile[] files = new CorpusFile[corpusFiles];
    BitReader bitReader = new BitReader(input);
    for (int i = 0; i < corpusFiles; ++i) {
      BitSet bitSet = new MyBitSet(wordListCount);
      for (int j = 0; j < wordListCount; ++j) {
        boolean value = bitReader.read();
        bitSet.set(j, value);
        if (value) {
          // compute each word's frequency as we go
          ++wordFrequency[j];
        }
      }
      files[i] = new CorpusFile(filePaths[i], bitSet);
    }

    Cluster topCluster = Cluster.create("root", files, wordList, wordFrequency);

    return new Corpus(indexCreationDate, corpusChars, corpusWords, indexDir,
        indexFileSize, varBreakChars, stopFileName, wordList, wordFrequency,
        files, topCluster);
  }

  public Corpus(long indexCreationDate, long corpusChars, long corpusWords,
      String indexDir, long indexFileSize, String varBreakChars,
      String stopFileName, String[] wordList, int[] wordFrequency,
      CorpusFile[] files, Cluster topCluster) {
    this.indexCreationDate = indexCreationDate;
    this.corpusChars = corpusChars;
    this.corpusWords = corpusWords;
    this.indexDir = indexDir;
    this.indexFileSize = indexFileSize;
    this.varBreakChars = varBreakChars;
    this.stopFileName = stopFileName;
    this.wordList = wordList;
    this.wordFrequency = wordFrequency;
    this.files = files;
    this.topCluster = topCluster;
  }

  public CorpusFile getCorpusFile(String fileName) {
    fileName = Util.toLowerCase(fileName);
    for (int i = 0; i < files.length; ++i) {
      CorpusFile corpusFile = files[i];
      String filePath = Util.toLowerCase(corpusFile.getFilePath());
      if (filePath.equals(fileName) || filePath.endsWith(fileName)) {
        return corpusFile;
      }
    }
    return null;
  }

  public long getIndexCreationDate() {
    return indexCreationDate;
  }

  public String getIndexText() {
    StringBuffer sb = new StringBuffer();
    sb.append("DTI.out");
    sb.append('\n');

    // print date of time of construction
    Date indexDate = new Date(indexCreationDate);
    sb.append("Created: " + indexDate);
    sb.append('\n');

    // print variable break characters
    sb.append("Variable break characters=" + varBreakChars);
    sb.append('\n');

    // print stop word file name
    if (stopFileName.length() > 0) {
      sb.append("Stop file: " + stopFileName);
    } else {
      sb.append("No stop words");
    }
    sb.append('\n');

    // print construction directory
    sb.append("Created in: " + indexDir);
    sb.append('\n');

    // print index file size
    sb.append("Index size: " + indexFileSize + " bytes");
    sb.append('\n');
    return sb.toString();
  }

  public String getStatsText() {
    return "Corpus stats: " + files.length + " files, " + corpusWords
        + " words, " + corpusChars + " characters";
  }

  public Cluster getTopCluster() {
    return topCluster;
  }

  public CorpusFile[] query(String[] args) {
    BitSet bitSet = new MyBitSet(wordList.length);
    for (int i = 0; i < args.length; ++i) {
      int foundIndex = Arrays.binarySearch(wordList, args[i]);
      if (foundIndex >= 0) {
        bitSet.set(foundIndex);
      }
    }
    if (bitSet.cardinality() == 0) {
      return new CorpusFile[0];
    }

    return search(bitSet, null);
  }

  public CorpusFile[] similarTo(CorpusFile base) {
    return search(base.getBitSet(), base);
  }

  private int computeSimilarity(BitSet bitSet, BitSet bitSet2) {
    // Compute matches by ANDing
    BitSet clone = (BitSet) bitSet.clone();
    clone.and(bitSet2);
    // Now add points for any matches
    int rating = 0;
    for (int i = clone.nextSetBit(0); i >= 0; i = clone.nextSetBit(i + 1)) {
      // The less frequent the word is, the more points its worth
      rating += files.length - wordFrequency[i];
    }
    return rating;
  }

  private CorpusFile[] search(BitSet bitSet, CorpusFile ignore) {
    Set matches = new TreeSet();
    for (int i = 0; i < files.length; ++i) {
      CorpusFile file = files[i];
      if (file != ignore) {
        int rating = computeSimilarity(bitSet, file.getBitSet());
        if (rating > 0) {
          matches.add(new Match(file, rating));
        }
      }
    }

    // Take up to the top 7
    int resultCount = Math.min(matches.size(), 7);
    CorpusFile[] results = new CorpusFile[resultCount];
    Iterator it = matches.iterator();
    for (int i = 0; i < resultCount; ++i) {
      results[i] = ((Match) it.next()).getFile();
    }
    return results;
  }

  private final long corpusChars;
  private final long corpusWords;
  private final CorpusFile[] files;
  private final long indexCreationDate;
  private final String indexDir;
  private final long indexFileSize;
  private final String stopFileName;
  private final Cluster topCluster;
  private final String varBreakChars;
  private final int[] wordFrequency;
  private final String[] wordList;

}
